import pandas as pd
start=pd.to_datetime("2014-01-01")
end=pd.to_datetime("2019-01-01")

import bs4 as bs 
import pickle
import requests

def save_sp500_tickers(path=''):
    """
    Return tickers  of S&P 500 and saves them in a pickle.
    
    Parameters path: path to save the ticker.
    """
    resp = requests.get("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies")
    soup = bs.BeautifulSoup(resp.text,'lxml')
    table = soup.find('table',{'class':"wikitable sortable"})
    tickers =[]
    for row in table.findAll('tr')[1:]:
        ticker = row.findAll('td')[0].text.replace("\n",'')
        ticker =ticker.replace(".","-")
        tickers.append(ticker)
    with open('sp500tickers.pickle','wb') as f:
        pickle.dump(tickers,f)
    return tickers
    

tickers = save_sp500_tickers()

import datetime as dt
import os
import pandas_datareader as web
import numpy as np
import quandl
import matplotlib.pyplot as plt
%matplotlib notebook

#Grabbing a bunch of tech stocks for our portfolio
aapl = quandl.get('WIKI/AAPL.11',start_date=start,end_date=end)
cisco = quandl.get('WIKI/CSCO.11',start_date=start,end_date=end)
ibm = quandl.get('WIKI/IBM.11',start_date=start,end_date=end)
amzn = quandl.get('WIKI/AMZN.11',start_date=start,end_date=end)

#saving data for future use
aapl.to_csv('AAPL_CLOSE')
cisco.to_csv('CISCO_CLOSE')
ibm.to_csv('IBM_CLOSE')
amzn.to_csv('AMZN_CLOSE')

def get_data(ticker,start,end,path,get_ticker=False,api ="yahoo"):
    """
    Save data of all S&P 500 companies.
    
    parameters 
    get_ticker= bolean, if true gets tickers.
    ticker: list of tickers
    path: string,path to save the data frames.
    start : tuple.start date of dataframe (year,month,day)
    end : tuple, start date of dataframe (year,month,day)
    api: string ,name of api to get data from.
    """
    i=0
    if get_ticker:
        tickers = save_sp500_tickers()
    if not os.path.exists(path):
        os.makedirs(path)
    start = dt.datetime(start)
    end = dt.datetime(end)
    for ticker in tickers:
        if not os.path.exists(path+'/{}.csv'.format(ticker)):
            df = web.DataReader(ticker,api,start,end)
            
            df.to_csv(path,'/{}.csv'.format(ticker))
        else:
            print("{}.csv Already exists".format(ticker))
        i+=1
        if i%20==0:
            print(str((i/len(tickers))*100)+" % completed") 

def compile_data(tickers,path):
    """
    Return a combined dataframe which include Adj. Close columns of all S&P 500 data frames
    parameters:
    tickers=list of tickers
    path : path of folder where data frames are saved.
    """
    main_df = pd.DataFrame()
    for count,ticker in enumerate(tickers):
        df= pd.read_csv(path+"/{}.csv".format(ticker))
        df.set_index("Date",inplace=True)
        df.rename(columns={"Adj Close":ticker},inplace=True)
        df.drop(['Open','High','Low','Close','Volume'],1,inplace=True)
        if main_df.empty:
            main_df=df
        else:
            main_df=main_df.join(df,how='outer')
        if count%100==0:
            print(str((count/len(tickers))*100)+" % completed")
    main_df.to_csv(path+'/sp500_joined_closes.csv')
    return main_df

def visulize_data(df,figsize):
    """
    returns a heatmap
    
    """
    df_corr=df.corr()
    data=df_corr.values
    fig = plt.figure(figsize=figsize)
    ax= fig.add_subplot(111)
    heatmap = ax.pcolor(data,cmap=plt.cm.RdYlBu)
    fig.colorbar(heatmap)
    ax.set_xticks(np.arange(data.shape[0]) + 0.5,minor=False)
    ax.set_yticks(np.arange(data.shape[1]) + 0.5,minor=False)
    ax.invert_yaxis()
    ax.xaxis.tick_top()
    column_labels = df_corr.columns
    row_labels=df_corr.index
    ax.set_xticklabels(column_labels)
    ax.set_yticklabels(row_labels)
    plt.xticks(rotation=90)
    heatmap.set_clim(-1,1)

df=pd.concat([aapl,cisco,ibm,amzn],1)
df.columns=['AAPl','CISCO','IBM','AMZN']

df.tail()

visulize_data(df,(5,5))

df[['AAPl','CISCO',"IBM","AMZN"]].pct_change(1).plot(kind='kde')

aapl['return']=((aapl['Adj. Close']/aapl['Adj. Close'].shift(1))-1)

aapl['return'].head()

aapl.fillna(0,inplace=True)
(aapl['return']+1).head()

aapl['Cumulative_returns']=(1+aapl['return']).cumprod()

aapl['Cumulative_returns'].head()

# doing same for all other stocks
for stock_df in (cisco,ibm,amzn):
    stock_df['return'] = (stock_df['Adj. Close']/stock_df['Adj. Close'].shift(1))-1
    stock_df.fillna(0,inplace=True)
    stock_df['Cumulative_returns']=(1+stock_df['return']).cumprod()

ibm.head()

for stock_df,allo in zip([aapl,cisco,ibm,amzn],[.3,.2,.4,.1]):
    stock_df['Allocation'] = stock_df['Cumulative_returns']*allo

aapl.head()

for stock_df in [aapl,cisco,ibm,amzn]:
    stock_df['Position Values'] = stock_df['Allocation']*1000000

portfolio_val = pd.concat([aapl['Position Values'],cisco['Position Values'],ibm['Position Values'],amzn['Position Values']],axis=1)
portfolio_val.columns = ['AAPL','CISCO','IBM','AMZN']
portfolio_val['Total Pos'] = portfolio_val.sum(axis=1)
portfolio_val.head()

fig=plt.figure()
portfolio_val['Total Pos'].plot(figsize=(8,5))
plt.title('Total Portfolio Value')

portfolio_val.drop('Total Pos',axis=1).plot(kind='line')

portfolio_val['Daily_return']=portfolio_val['Total Pos'].pct_change(1)

sr=portfolio_val['Daily_return'].mean()/portfolio_val['Daily_return'].std()

sr

portfolio_val['Daily_return'].head()

asr=(225**0.5)*sr

asr

stocks = pd.concat([aapl["Adj. Close"],cisco["Adj. Close"],ibm["Adj. Close"],amzn["Adj. Close"]],axis=1)
stocks.columns = ['aapl','cisco','ibm','amzn']

stocks.head()

#average daily reurn
stocks.pct_change(1).mean()

#average yearly return
stocks.pct_change(1).mean()*252

stocks.pct_change(1).corr()

visulize_data(stocks.pct_change(1),(6,6))

stock_normed=stocks/stocks.iloc[0]
stock_normed.plot();

stock_daily_return=stocks.pct_change(1)
stock_daily_return.head()

log_return=np.log(stocks)-np.log(stocks.shift(1))

log_return.hist(bins=100,figsize=(12,6));
plt.tight_layout()

log_return.describe().T

log_return.cov()

visulize_data(log_return,(6,6))

np.random.seed(101)

def monte_carlo_simulation(num_iter,log_ret):
    """
    returns sharpe_ratio array,weights array,volatility array and yearly_log_return array
    parameters:
    num_iter: Number of itterations
    log_ret=log return data frame of stocks
    
    """
    weights_mat=np.zeros((num_iter,len(log_return.columns)))
    returns=np.zeros(num_iter)
    volatility = np.zeros(num_iter)
    sharpe_ratio=np.zeros(num_iter)
    for ind in range(num_iter):
        weights=np.array(np.random.random(len(weights_mat[0])))
        weights=weights/np.sum(weights)
        weights_mat[ind,:]=weights
        returns[ind]=np.sum((log_ret.mean() * weights) *252)
        volatility[ind]=np.sqrt(np.dot(weights.T, np.dot(log_ret.cov() * 252, weights)))
        sharpe_ratio[ind]=returns[ind]/volatility[ind]
        
    print("Max Sharpe ratio: "+str(sharpe_ratio.max()))
    ind=sharpe_ratio.argmax()
    print("Weights: ")
    for i in range(len(log_return.columns)):
        print(str(log_return.columns[i])+": "+str(weights_mat[ind][i]))
    return sharpe_ratio,weights_mat,volatility,returns
    

sr,weights,volatility,returns=monte_carlo_simulation(10000,log_return)

plt.figure(figsize=(8,8))
plt.scatter(volatility,returns,c=sr,cmap='plasma')
plt.colorbar(label='Sharpe Ratio')
plt.xlabel('Volatility')
plt.ylabel('Return')

# Add red dot for max SR
plt.scatter(volatility[sr.argmax()],returns[sr.argmax()],c='red',s=50,edgecolors='black')

from scipy.optimize import minimize

def get_weights(num_weights,num_iter):
    weights_mat=np.zeros((num_iter,num_weights))
    for ind in range(num_iter):
        weights=np.array(np.random.random(len(weights_mat[0])))
        weights=weights/np.sum(weights)
        weights_mat[ind,:]=weights
    
    return weights_mat

weights=get_weights(4,10000)

def get_ret_vol_sr(weights,log_return):
    """
    returns array of yearly returns ,volatility,sharpe ratio ,"""
    ret= np.sum(log_return.mean() * weights) * 252
    vol = np.sqrt(np.dot(weights.T, np.dot(log_return.cov() * 252, weights)))
    sr = ret/vol
    return np.array([ret,vol,sr])
    

def neg_sharpe(weights):
    return get_ret_vol_sr(weights,log_return)[2]*-1

def check_constraint(weights):
    """return 0 if constraint is satisfied"""
    return np.sum(weights)-1

# By convention of minimize function it should be a function that returns zero for conditions
cons = ({'type':'eq','fun': check_constraint})

# adding bounds for each weights
bounds=((0,1),(0,1),(0,1),(0,1))

#intial weights 
init=[2.5,2.5,2.5,2.5]

result=minimize(neg_sharpe,init,method="SLSQP",bounds=bounds,constraints=cons)

result

result.x

get_ret_vol_sr(result.x,log_return)

# Max Return is 0.23  so keeping range between 0-0.3
frontier_y=np.linspace(0,0.3,100)

def minimize_volatility(weights):
    return  get_ret_vol_sr(weights,log_return)[1] 

frontier_volatility = []

for possible_return in frontier_y:
    # function for return
    cons = ({'type':'eq','fun': check_constraint},
            {'type':'eq','fun': lambda w: get_ret_vol_sr(w,log_return)[0] - possible_return})
    
    result = minimize(minimize_volatility,init,method='SLSQP',bounds=bounds,constraints=cons)
    
    frontier_volatility.append(result['fun'])

plt.figure(figsize=(8,8))
plt.scatter(volatility,returns,c=sr,cmap='plasma')
plt.colorbar(label='Sharpe Ratio')
plt.xlabel('Volatility')
plt.ylabel('Return')

# Add frontier line
plt.plot(frontier_volatility,frontier_y,'g--',linewidth=3)

from scipy import stats

spy = web.DataReader('SPY','yahoo')

spy.info()

spy.tail()

start = pd.to_datetime('2015-02-02')
end = pd.to_datetime('2020-01-31')

start = pd.to_datetime('2015-02-02')
end = pd.to_datetime('2020-01-31')
aapl = web.DataReader('AAPL','yahoo',start,end)

aapl.tail()

aapl['Cumulative'] = aapl['Close']/aapl['Close'].iloc[0]
spy['Cumulative'] = spy['Close']/spy['Close'].iloc[0]

plt.figure()
aapl['Cumulative'].plot(label='AAPL',figsize=(8,8))
spy['Cumulative'].plot(label='SPY Index')
plt.legend()
plt.title('Cumulative Return')

aapl['Daily Return'] = aapl['Close'].pct_change(1)
spy['Daily Return'] = spy['Close'].pct_change(1)

plt.figure()
plt.scatter(aapl['Daily Return'],spy['Daily Return'],alpha=0.3)

plt.figure()
aapl['Daily Return'].hist(bins=100,label="AAPL");
spy['Daily Return'].hist(bins=100,label="SPY",alpha=0.6);

beta,alpha,r_value,p_value,std_err = stats.linregress(aapl['Daily Return'].iloc[1:],spy['Daily Return'].iloc[1:])

print("Beta: {} \nAlpha : {} \nR Value : {} \nP Value: {} \nStd:{}".format(beta,alpha,r_value,p_value,std_err))

noise = np.random.normal(0,0.001,len(spy['Daily Return'].iloc[1:]))

beta,alpha,r_value,p_value,std_err = stats.linregress(spy['Daily Return'].iloc[1:]+noise,spy['Daily Return'].iloc[1:])

print("Beta: {} \nAlpha : {} \nR Value : {} \nP Value: {} \nStd:{}".format(beta,alpha,r_value,p_value,std_err))

import matplotlib.dates as mdates
import mpl_finance as finance

df_ohlc = aapl['Close'].resample('10D').ohlc()
df_vol = aapl["Volume"].resample("10D").sum()
df_ohlc.reset_index(inplace=True)
df_ohlc["Date"]=df_ohlc["Date"].map(mdates.date2num)

fig=plt.figure(figsize=(15,6))
ax1=plt.subplot2grid((8,1),(0,0),rowspan=6,colspan=1)
ax2=plt.subplot2grid((8,1),(7,0),rowspan=2,colspan=1,sharex=ax1)
ax1.xaxis_date()
finance.candlestick_ohlc(ax1,df_ohlc.values,width=2,colorup='g')
ax2.fill_between(df_vol.index.map(mdates.date2num),df_vol.values,0)

import plotly.graph_objects as go

fig = go.Figure(data=[go.Candlestick(x=aapl.index,
                open=aapl['Open'], high=aapl['High'],
                low=aapl['Low'], close=aapl['Close'])
                      ])
fig.update_layout(xaxis_rangeslider_visible=False)
fig.show()

fig.write_html("Candlestick1.html")

#with X_axis Slider
fig = go.Figure(data=[go.Candlestick(x=aapl.index,
                open=aapl['Open'], high=aapl['High'],
                low=aapl['Low'], close=aapl['Close'])
                      ])
fig.update_layout(
    title='Candlestick',
    yaxis_title='AAPL Stock',
    shapes = [dict(
        x0='2016-12-09', x1='2016-12-09', y0=0, y1=1, xref='x', yref='paper',
        line_width=2)],
    annotations=[dict(
        x='2016-12-09', y=0.05, xref='x', yref='paper',
        showarrow=False, xanchor='left', text='Increase Period Begins')]
)
fig.show()

fig.write_html("Candlestick2.html")

from statsmodels.tsa.stattools import adfuller

# Store in a function for later use!
def adf_check(time_series):
    """
    Function to return formated adfuller test results 
    Parameters time series 
    """
    result = adfuller(time_series)
    print('Augmented Dickey-Fuller Test:')
    labels = ['ADF Test Statistic','p-value','#Lags Used','Number of Observations Used']

    for value,label in zip(result,labels):
        print(label+' : '+str(value) )
    
    if result[1] <= 0.05:
        print("strong evidence against the null hypothesis, reject the null hypothesis. Data has no unit root and is stationary")
    else:
        print("weak evidence against null hypothesis, time series has a unit root, indicating it is non-stationary ")

start = pd.to_datetime('2015-02-02')
end = pd.to_datetime('2020-01-31')
aapl = web.DataReader('AAPL','yahoo',start,end)

df=pd.DataFrame(aapl.Close)

df=df.resample(rule="B").mean()

for i in range(len(df)):
    if np.isnan(df["Close"][i]):
        df["Close"][i]=np.mean((df['Close'][i+1],df['Close'][i-1]))

adf_check(df['Close'])

df['First Difference'] = df['Close'] - df['Close'].shift(1)

df['Second Difference'] = df['First Difference'] - df['First Difference'].shift(1)

adf_check(df['First Difference'].dropna())

adf_check(df['Second Difference'].dropna())

df['Seasonal Diffrence']=df["Close"]-df["Close"].shift(12)

adf_check(df['Seasonal Diffrence'].dropna())

from statsmodels.graphics.tsaplots import plot_acf,plot_pacf

fig=plot_acf(df['First Difference'].dropna())

fig=plot_pacf(df["First Difference"].dropna())

fig=plot_acf(df['Second Difference'].dropna())

fig=plot_pacf(df["Second Difference"].dropna())

fig=plot_acf(df['Seasonal Diffrence'].dropna())

fig=plot_pacf(df["Seasonal Diffrence"].dropna())

from statsmodels.tsa.arima_model import ARIMA
import statsmodels.api as sm
import warnings
warnings.filterwarnings('ignore')

model_ns=ARIMA(df["Close"],(1,1,1),dates=df.index,freq='B')
results_ns = model_ns.fit()
print(results_ns.summary())

results_ns.resid.plot(figsize=(10,5))

results_ns.resid.plot(kind="kde")

model = sm.tsa.statespace.SARIMAX(df['Close'],order=(0,1,0), seasonal_order=(1,1,1,24),dates=df.index,freq='B')
results = model.fit()
print(results.summary())

results.resid.plot(figsize=(10,5))

results.resid.plot(kind="kde")

df['Predict'] = results.predict(start = 1240, end= 1305, dynamic= True)  
df[['Close','Predict']][1240:1305].plot(figsize=(6,4));

import pandas as pd
import numpy as np
from sklearn.svm import SVR
import matplotlib.pyplot as plt

aapl.head()

dates=aapl.index[10:-40]

data=np.array(aapl["Close"])

y=data[10:-9]

x=np.zeros((len(y),10))
for i in range(10,len(y)+10):
    x[i-10]=data[i-10:i]  

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = x[:int(len(x)*0.75)], x[int(len(x)*0.75):],y[:int(len(x)*0.75)],y[int(len(x)*0.75):]

svr_rbf = SVR(kernel='rbf', C=1e3, gamma=0.1,verbose=True)

svr_rbf.fit(X_train,y_train)

svr_rbf.score(X_test,y_test)

from sklearn.neural_network import MLPRegressor

model=MLPRegressor((50,100,50),batch_size=16,learning_rate='adaptive',early_stopping=True,warm_start=True,verbose=True)

model.fit(X_train,y_train)

model.score(X_test,y_test)

ann_df=pd.DataFrame()
ann_df['Data']=y_test
ann_df["ANN Predict"]=model.predict(X_test)
ann_df.index=dates[-len(y_test):]

fig=go.Figure()
fig.add_trace(go.Scatter(x=ann_df.index, y=ann_df['Data'], mode='lines',
        name="Data",
        line=dict(color='rgb(115,115,115)', width=3),
        connectgaps=True,
    ))

fig.add_trace(go.Scatter(x=ann_df.index, y=ann_df['ANN Predict'], mode='lines',
        name="Predict",
        line=dict(color='rgb(49,130,189)', width=2),
        connectgaps=True,
    ))
fig.show()

fig.write_html("ANN.html")

loss=model.loss_curve_
num_iter=range(model.n_iter_)
val_score=model.validation_scores_

plt.figure()
plt.plot(num_iter,loss,label="Loss")
plt.legend()

plt.figure()
plt.plot(num_iter,val_score,label="Score")
plt.legend()

from keras.models import Sequential
from keras.layers import Dense, LSTM
import keras
import keras.optimizers as optim 

X_train=X_train.reshape((X_train.shape[0],X_train.shape[1],1))                   

start = pd.to_datetime('2009-02-02')
end = pd.to_datetime('2020-01-31')

aapl = web.DataReader("aapl","yahoo",start,end)

data = aapl.Close.values

from sklearn.preprocessing import MinMaxScaler

scaler = MinMaxScaler(feature_range=(0,1))
scaled_data = scaler.fit_transform(data.reshape(-1,1))

data= scaled_data.squeeze()

y=data[60:-40]

x=np.zeros((len(y),60))
for i in range(60,len(y)+60):
    x[i-60]=data[i-60:i]  

X_train, X_test, y_train, y_test = x[:int(len(x)*0.75)], x[int(len(x)*0.75):],y[:int(len(x)*0.75)],y[int(len(x)*0.75):]

X_train=X_train.reshape((X_train.shape[0],X_train.shape[1],1))                   

X_test=X_test.reshape((X_test.shape[0],X_test.shape[1],1))                   

X_train.shape

model = Sequential()
model.add(LSTM(50, return_sequences=True, input_shape= (X_train.shape[1], 1)))
model.add(LSTM(50, return_sequences= False))
model.add(Dense(25))
model.add(Dense(1))
model.compile(optimizer=optim.Adam(learning_rate=0.001), loss='mean_squared_error')

history=model.fit(X_train, y_train, batch_size=1, epochs=5,validation_split=0.1)

actual_data = aapl.Close.values[60:-40]

y_test = actual_data[int(len(x)*0.75):]

df=pd.DataFrame()

X_test.shape, X_train.shape

df["predict"]=scaler.inverse_transform(model.predict(X_test)).reshape(len(y_test))
df['data']=y_test
df.index=dates[-len(y_test):]

fig=go.Figure()
fig.add_trace(go.Scatter(x=df.index, y=df['data'], mode='lines',
        name="Data",
        line=dict(color='rgb(115,115,115)', width=3),
        connectgaps=True,
    ))

fig.add_trace(go.Scatter(x=df.index, y=df['predict'], mode='lines',
        name="Predict",
        line=dict(color='rgb(49,130,189)', width=2),
        connectgaps=True,
    ))
fig.show()

fig.write_html("LSTM.html")

history.history